
import Crawler from 'crawler';
import fs from 'fs';

const ws = fs.createWriteStream("links-all.txt",{

});


const d = new Crawler({
    encoding: null,
    jQuery: true,// set false to suppress warning message.
    rateLimit: 200,
    maxConnections: 1,
    callback: (err, res, done) => {
        if (err) {
            console.error(err.stack);
        } else {
            let a = res.$('.field-item .file a');
            
            let all = a.get();
            for (let i = 0;  i < all.length; i+=1) {
                let e = all[i];
                //console.log(e.attribs['href']);
                ws.write(e.attribs['href']+"\n");
            }
        }
        done()
    }

});


const c = new Crawler({
    encoding: null,
    jQuery: true,// set false to suppress warning message.
    rateLimit: 200,
    maxConnections: 1,
    callback: (err, res, done) => {
        if (err) {
            console.error(err.stack);
        } else {
            const $ = res.$;
            let content = $('#block-system-main');
            let a = content.find('.field-item a');
            let all = a.get();
            for (let i = 0;  i < all.length; i+=2) {
                let e = all[i];
                //console.log(e.attribs['href']);
                d.queue({
                    uri: 'https://opengameart.org' + e.attribs['href']
                });
                
            }
        }

        done();
    }
});




for (let i = 190; i < 191; i++) {
    c.queue({
        uri: 'https://opengameart.org/art-search-advanced?keys=&field_art_type_tid%5B0%5D=10&sort_by=count&sort_order=DESC&page=' + i,
        limiter: 'proxy_1',
        proxy: 'proxy_1'
    });
}
